{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('display.max_colwidth', 60)#使得再长的名字也能全部显示\n",
    "pd.set_option('display.max_columns', 200)\n",
    "pd.set_option('display.max_rows', 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def non_analysis(path, zh, reference):\n",
    "    #分析表格缺省值情况\n",
    "    df = pd.read_csv(path)\n",
    "    #base_data.info()   #显示缺省值\n",
    "    \n",
    "    missing = df.isnull().sum().values/len(df)\n",
    "    if zh:\n",
    "        new_df = pd.DataFrame([zh, missing]).T\n",
    "        new_df.columns = ['zh_name', 'None_percent']\n",
    "    else:\n",
    "        new_df = pd.DataFrame([missing]).T\n",
    "        new_df.columns = ['None_percent']\n",
    "    new_df.index = df.columns.values\n",
    "    #new_df['None_percent']=df['None_percent'].apply(lambda x:'%.4f'%x)   #转换成百分比\n",
    "    \n",
    "    if reference:\n",
    "        new_df['reference']=[False]*len(new_df)\n",
    "        for p in reference:\n",
    "            new_df.loc[p, 'reference'] = True\n",
    "    \n",
    "    new_df['example'] = [None]*len(new_df)\n",
    "    for _ in new_df.index:\n",
    "        p = df[~(df[_].isnull())][_]\n",
    "        if len(p)!=0:\n",
    "            new_df.loc[_, 'example']=p.iloc[0]\n",
    "    \n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path=\"train/tax_info.csv\"\n",
    "# drop = ['enttypeitem', 'opto', 'empnum', 'compform', 'parnum',\n",
    "#        'exenum', 'opform', 'ptbusscope', 'venind', 'enttypeminu',\n",
    "#        'midpreindcode', 'protype', 'reccap', 'forreccap',\n",
    "#        'forregcap', 'congro', 'dom', 'opscope', 'oploc', 'id']\n",
    "str = \"id:企业唯一标识, START_DATE:起始时间, END_DATE:终止时间, TAX_CATEGORIES:税种, TAX_ITEMS:税目, TAXATION_BASIS:计税依据, TAX_RATE:税率, DEDUCTION:扣除数, TAX_AMOUNT:税额\"\n",
    "p = str.split(',')\n",
    "zh_name = []\n",
    "for _ in p:\n",
    "    zh_name.append(_.split(':')[1])\n",
    "\n",
    "df = non_analysis(path, zh_name, [])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>zh_name</th>\n",
       "      <th>None_percent</th>\n",
       "      <th>example</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <td>企业唯一标识</td>\n",
       "      <td>0</td>\n",
       "      <td>f000950527a6feb6c2f40c9d8477e73a439dfa0897830397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>START_DATE</th>\n",
       "      <td>起始时间</td>\n",
       "      <td>0</td>\n",
       "      <td>2015/09/01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>END_DATE</th>\n",
       "      <td>终止时间</td>\n",
       "      <td>0</td>\n",
       "      <td>2015/09/30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TAX_CATEGORIES</th>\n",
       "      <td>税种</td>\n",
       "      <td>0</td>\n",
       "      <td>印花税</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TAX_ITEMS</th>\n",
       "      <td>税目</td>\n",
       "      <td>0</td>\n",
       "      <td>工伤保险（单位）</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TAXATION_BASIS</th>\n",
       "      <td>计税依据</td>\n",
       "      <td>0.115739</td>\n",
       "      <td>72530.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TAX_RATE</th>\n",
       "      <td>税率</td>\n",
       "      <td>0.115739</td>\n",
       "      <td>0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DEDUCTION</th>\n",
       "      <td>扣除数</td>\n",
       "      <td>0.169892</td>\n",
       "      <td>-0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TAX_AMOUNT</th>\n",
       "      <td>税额</td>\n",
       "      <td>0</td>\n",
       "      <td>21.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               zh_name None_percent  \\\n",
       "id              企业唯一标识            0   \n",
       "START_DATE        起始时间            0   \n",
       "END_DATE          终止时间            0   \n",
       "TAX_CATEGORIES      税种            0   \n",
       "TAX_ITEMS           税目            0   \n",
       "TAXATION_BASIS    计税依据     0.115739   \n",
       "TAX_RATE            税率     0.115739   \n",
       "DEDUCTION          扣除数     0.169892   \n",
       "TAX_AMOUNT          税额            0   \n",
       "\n",
       "                                                         example  \n",
       "id              f000950527a6feb6c2f40c9d8477e73a439dfa0897830397  \n",
       "START_DATE                                            2015/09/01  \n",
       "END_DATE                                              2015/09/30  \n",
       "TAX_CATEGORIES                                               印花税  \n",
       "TAX_ITEMS                                               工伤保险（单位）  \n",
       "TAXATION_BASIS                                           72530.8  \n",
       "TAX_RATE                                                  0.0003  \n",
       "DEDUCTION                                                  -0.04  \n",
       "TAX_AMOUNT                                                  21.8  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# base_info.csv\n",
    "opscope  企业类型需要重新标识\n",
    "\n",
    "enttypeitem   企业类型小类不需要删去\n",
    "\n",
    "empnum\t从业人数不需要删去\n",
    "\n",
    "venind\t风险行业不删去\n",
    "\n",
    "reccap\t实缴资本看情况"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# change_info.csv\n",
    "bgxmdm属性为数字，有作为属性使用的可能 也可以直接将整个表整合为企业有无发生变更次数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# annual_report_info.csv\n",
    "FUNDAM   资金数额\n",
    "以及其它人数相关的将NON改为0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "drop=[\"ANCHEYEAR\", \"MEMNUM\", \"FARNUM\", \"ANNNEWMEMNUM\", \"ANNREDMEMNUM\"]\n",
    "\n",
    "fill=[\"FUNDAM\", \"COLGRANUM\", \"RETSOLNUM\", \"DISPERNUM\", \"UNENUM\", \"COLEMPLNUM\", \"RETEMPLNUM\", \"DISEMPLNUM\", \"UNEEMPLNUM\"]\n",
    "\n",
    "cate_feat=[\"STATE\", \"BUSSTNAME\", \"EMPNUMSIGN\", \"WEBSITSIGN\", \"FORINVESTSIGN\", \"STOCKTRANSIGN\", \"PUBSTATE\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tax_info.csv\n",
    "建议写成  税种：税额的特征形式"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# news_info.csv\n",
    "建议写成 积极数 消极数 中立数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# other_info.csv\n",
    "所有的缺失值改为0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
